package com.blz.flare.crawler;

import au.id.jericho.lib.html.MasonTagTypes;
import au.id.jericho.lib.html.MicrosoftTagTypes;
import au.id.jericho.lib.html.PHPTagTypes;
import com.blz.flare.data.SerializedQueue;
import org.slf4j.LoggerFactory;

import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.Collections;
import java.util.HashSet;
import java.util.Queue;
import java.util.Set;

/**
 * Thread that handles crawling.
 *
 * @author pchapman
 * @author jeremy.arnold
 */
//TODO add method to request that the crawler be stopped
public class Crawler extends Thread {

	private CrawlerConfig config;
	private ThreadGroup crawlerThreads;

    /**
     * Construct the {@link Crawler} with a specified {@link CrawlerConfig}.
     * @param config the configuration
     */
	public Crawler(CrawlerConfig config) {
		super("Flare Website Crawler");
		setDaemon(true);
		this.config = config;
		MicrosoftTagTypes.register();
		PHPTagTypes.register();
		PHPTagTypes.PHP_SHORT.deregister(); // remove PHP short tags
		MasonTagTypes.register();
	}

    /**
     * {@inheritDoc}
     */
	public void run() {
		// Initialize the URIs to index
		Queue<URI> URIs = new SerializedQueue<URI>(config.getStartURIs());
		Set<URI> processed = Collections.synchronizedSet(new HashSet<URI>());
		// Set up the crawler threads
		try {
			crawlerThreads = new ThreadGroup("Flare Website Crawler Thread Group");
			Thread t;
			for (int i = 0; i < config.getThreadCount(); i++) {
				t = new CrawlerThread(crawlerThreads, config, URIs, processed);
				t.start();
				try {
					Thread.sleep(30000L);	// Wait on the queue to be loaded with a few items.
				} catch (InterruptedException e) {}
			}
		} catch (MalformedURLException e) {
			LoggerFactory.getLogger(getClass()).error("Unable to configure crawler threads", e);
			crawlerThreads = null;
		}
		// Wait for all threads to finish
		while (crawlerThreads.activeCount() > 0) {
			try {
				sleep(500L);
			} catch (InterruptedException e) {}
		}
		crawlerThreads = null;
	}
}
